// 1.	express启动一个简单的Http服务
// 2.	分析目标页面DOM结构，找到所要抓取的信息的相关DOM元素
// 3.	使用superagent请求目标页面
// 4.	使用cheerio获取页面元素，获取目标数据
// 5.   将数据储存到monggodb，使chapter储存cover的bookid
// 6.   使用关联查询将书封和章节能用同一id返回到前端
// 7.   将chapter里的id传入content里 
// 8.   根据chapterid 查找获取章节详情内容再返回前端

// 当前查重
// 
// aaa为爬虫方法
// bbb为储存数据方法


// 方案1：
// 每次都储存最新章节的名称
// 做对比更新(问题：只能发现最新第一章)

// 方案2：
// 每次重新抓取（问题：可能会被封ip）

// 方案3：
// 用所以章节数量做对比(问题：还是相当于第二种，因为获取当前总章节还是要爬取所有)
const express = require('express');
const app = express();

let server = app.listen(3000, function () {
    let host = "192.168.1.7";
    let port = server.address().port;
    console.log('Your App is running at http://%s:%s', host, port);
});
const superagent = require('superagent');

let hotNews = [];
let localNews = [];

function aaa(a, b, c, d) {
    var bookId = [];
    superagent.get(a).end((err, res) => {
        if (err) {
            console.log(`小说抓取失败 - ${err}`)
        } else {
            hotNews = getHotNews(res)
            // 做书名判断

            // 获取书名
            for (var variable in hotNews) {
                // 这里将书封简介的id传到章节

                superagent.get(hotNews[variable].href).end((err, res) => {
                    if (err) {
                        console.log(`小说抓取失败 - ${err}`)
                    } else {
                        hotNews = getHotNews(res)
                        for (let i = 0; i <= hotNews.length; i++) {
                            var obj = Object.assign(hotNews[0], hotNews[1]);
                            // Object.assign({}, config)
                        }
                        // console.log(chapter)

                        // 存简介
                        var surfaceName = "cover"
                        bbb(obj, surfaceName)
                        app.get('/', function (req, res) {
                            res.send(hotNews);
                        });
                    }
                });
                let getHotNews = (res) => {
                    let hotNews = [];
                    let $ = cheerio.load(res.text);
                    $(c).each((idx, ele) => {
                        let news = {
                            name: $(ele).text(),
                            // href: $(ele).attr('href')
                        };
                        hotNews.push(news)
                    });
                    $(d).first().each((idx, ele) => {
                        var authors = $(ele).text();
                        let news = {
                            author: authors.replace(/\s/g, "").substring(3, authors.length),
                            create_time: "" + (new Date()).getTime() * 1,
                            update_time: "" + (new Date()).getTime() * 1,
                            // href: $(ele).attr('href')
                        };
                        hotNews.push(news)
                    });
                    return hotNews
                };
            }
            app.get('/', function (req, res) {
                res.send(hotNews);
            });
        }
    });
    const cheerio = require('cheerio');

    let getHotNews = (res) => {
        let hotNews = [];
        let $ = cheerio.load(res.text);
        $(b).each((idx, ele) => {
            let news = {
                title: $(ele).text(),
                href: $(ele).attr('href'),
                bookId: bookId[idx]
            };
            hotNews.push(news)
        });
        return hotNews
    };
}
// 网址
var webURL = 'http://www.xbiquge.la/xuanhuanxiaoshuo/';

// 第一层书籍路径
var way = 'div.r ul li span a';

// 第二层章节路径>获取name
var url = 'div#info h1';
// 第二层章节路径>获取author
var author = 'div#info p';


// 获取书封
aaa(webURL, way, url, author)
function bbb(msg, surfaceName) {
    var MongoClient = require("mongodb").MongoClient;
    var DB_URL = "mongodb://localhost:27017/chm";

    function insertData(db) {
        // monggo自带语句删除同一name项
        
        // doc.dups.shift();
        // db.cover.remove({ _id: { $in: doc.dups } });
        var devices = db.collection(surfaceName);
        var data = msg;
        devices.insert(data, function (error, result) {
            if (error) {
                console.log('Error:' + error);
            } else {
                // console.log(result.result.n);
            }
            db.close();
        });
    }

    MongoClient.connect(DB_URL, function (error, db) {
        // console.log('连接成功!');
        insertData(db);
    });
}

function find(msg, surfaceName, callback) {

    var MongoClient = require("mongodb").MongoClient;
    var DB_URL = "mongodb://localhost:27017/chm";

    function findData(db) {
        var devices = db.collection(surfaceName);
        devices.findOne(msg, function (error, result) {
            if (error) {
                console.log('Error:' + error);
            } else {
                var results = new Object(result)
                // console.log(results._id)
                callback(results._id)
            }
            db.close();
        });
    }

    MongoClient.connect(DB_URL, function (error, db) {
        console.log('连接成功!' + error);
        findData(db);
    });
}






// 获取章节
function ObtainChapter() {
    var bookId;
    superagent.get("http://www.xbiquge.la/xuanhuanxiaoshuo/").end((err, res) => {
        if (err) {
            console.log(`小说抓取失败 - ${err}`)
        } else {
            hotNews = getHotNews(res)
            for (var variable in hotNews) {
                var ObjectID = require('mongodb').ObjectID;
                var ObjectIDa = new ObjectID()
                bookId = ObjectIDa
                // 获取到书封id
                // bookId.push(ObjectIDa)
                // console.log(bookId)
                // console.log(hotNews[variable].href)
                superagent.get(hotNews[variable].href).end((err, res) => {
                    if (err) {
                        console.log(`小说抓取失败 - ${err}`)
                    } else {
                        hotNews = getHotNews(res, function (hotNews) {

                            // console.log(hotNews)
                            // 存章节
                            var surfaceName = "chapter"
                            bbb(hotNews, surfaceName)


                            app.get('/', function (req, res) {
                                res.send(hotNews);
                            });
                        })
                    }
                });
                let getHotNews = (res, callback1) => {
                    let hotNews = [];
                    let $ = cheerio.load(res.text);
                    $("div#info h1").each((idx, ele) => {
                        var book;
                        let news = {
                            name: $(ele).text(),
                        };

                        find(news, "cover", function (id) {
                            // console.log("===================== " + id)
                            // 储存id
                            $("div#list dl dd a").each((idx, ele) => {
                                let news = {
                                    chapters: $(ele).text(),
                                    href: $(ele).attr('href'),
                                    book_id: id,
                                    create_time: "" + (new Date()).getTime() * 1,
                                    update_time: "" + (new Date()).getTime() * 1,
                                };
                                hotNews.push(news)
                            });
                            callback1(hotNews)
                        })
                    });
                };
            }
            app.get('/', function (req, res) {
                res.send(hotNews);
            });
        }
    });
    const cheerio = require('cheerio');
    let getHotNews = (res) => {
        let hotNews = [];
        let $ = cheerio.load(res.text);
        $("div.r ul li span a").each((idx, ele) => {
            let news = {
                name: $(ele).text(),
                href: $(ele).attr('href')
            };
            hotNews.push(news)
        });
        return hotNews
    };
}
setTimeout(() => {
    ObtainChapter()
}, 500)
